library(NNbenchmark)
library(ANN2)
library(stringr)
library(dplyr)
library(kableExtra)
Set Environment
options(scipen = 9999)
options("digits.secs" = 2)
timer <- createTimer(verbose = FALSE)
Datasets to Test
NNdataSummary(NNdatasets)
NN Train Function
hyperParams <- function(optim_method) {
if (!is.element(optim_method, c("sgd", "adam", "rmsprop"))) stop("Invalid Parameters.")
if (optim_method == "sgd") { iter <- 10; lr <- 0.01; params <- paste0("method=", optim_method, "_", "lr=", lr, "_", "iter=", iter)}
if (optim_method == "adam") { iter <- 15; lr <- 0.02; params <- paste0("method=", optim_method, "_", "lr=", lr, "_", "iter=", iter)}
if (optim_method == "rmsprop") { iter <- 20; lr <- 0.03; params <- paste0("method=", optim_method, "_", "lr=", lr, "_", "iter=", iter)}
params <- paste0("method=", optim_method, "_", "lr=", lr, "_", "iter=", iter)
out <- list(iter = iter, lr = lr, params = params)
return (out)
}
NNtrain <- function(x, y, hidden_neur, optim_method) {
hyper_params <- hyperParams(optim_method)
iter <- hyper_params$iter
lr <- hyper_params$lr
NNreg <- neuralnetwork(X = x, y = y,
val.prop = 0,
standardize = FALSE,
hidden.layers = c(hidden_neur),
regression = TRUE,
loss.type = "squared",
n.epochs = iter,
optim.type = optim_method,
learn.rates = lr,
verbose = FALSE,
random.seed = as.integer(runif(1)*10000000))
return (NNreg)
}
Main Loop
for (dset in names(NNdatasets)) {
##Â =============================================
##Â EXTRACT INFORMATION FROM THE SELECTED DATASET
##Â =============================================
ds <- NNdatasets[[dset]]$ds
Z <- NNdatasets[[dset]]$Z
neur <- NNdatasets[[dset]]$neur
nparNN <- NNdatasets[[dset]]$nparNN
fmlaNN <- NNdatasets[[dset]]$fmlaNN
donotremove <- c("dset", "dsets", "ds", "Z", "neur", "TF", "nrep", "timer",
"donotremove", "donotremove2")
donotremove2 <- c("dset", "dsets")
##Â ===================================================
## SELECT THE FORMAT REQUIRED BY THE PACKAGE/ALGORITHM
## d = data.frame, m = matrix, v = vector/numeric
##Â ATTACH THE OBJECTS CREATED (x, y, Zxy, ... )
## ===================================================
ZZ <- prepareZZ(Z, xdmv = "d", ydmv = "v", zdm = "d", scale = TRUE)
attach(ZZ)
##Â =============================================
##Â SELECT THE PACKAGE USED FOR TRAINING
## nrep => SELECT THE NUMBER OF INDEPENDANT RUNS
##Â iter => SELECT THE MAX NUMBER OF ITERATIONS
##Â TF => PLOT THE RESULTS
##Â =============================================
nrep <- 5
TF <- TRUE
method <- c("sgd", "adam", "rmsprop")
for (m in method) {
descr <- paste(dset, "ANN2::neuralnetwork", m, sep = "_")
##Â AUTO
Ypred <- list()
Rmse <- numeric(length = nrep)
Mae <- numeric(length = nrep)
for(i in 1:nrep){
event <- paste0(descr, sprintf("_%.2d", i))
timer$start(event)
#### ADJUST THE FOLLOWING LINES TO THE PACKAGE::ALGORITHM
hyper_params <- hyperParams(optim_method = m)
NNreg <- tryCatch(
NNtrain(x = x, y = y, hidden_neur = neur, optim_method = m),
error = function(y) {lm(y ~ 0, data = Zxy)}
)
y_pred <- tryCatch(
ym0 + ysd0*predict(NNreg, x)$predictions,
error = ym0
)
####
Ypred[[i]] <- y_pred
Rmse[i] <- funRMSE(y_pred, y0)
Mae[i] <- funMAE(y_pred, y0)
timer$stop(event, RMSE = Rmse[i], MAE = Mae[i], params = hyper_params$params, printmsg = FALSE)
}
best <- which(Rmse == min(Rmse, na.rm = TRUE))[1]
best ; Rmse[[best]]
## ================================================
##Â PLOT ALL MODELS AND THE MODEL WITH THE BEST RMSE
##Â par OPTIONS CAN BE IMPROVED FOR A BETTER DISPLAY
## ================================================
op <- par(mfcol = c(1,2))
plotNN(xory, y0, uni, TF, main = descr)
for (i in 1:nrep) lipoNN(xory, Ypred[[i]], uni, TF, col = i, lwd = 1)
plotNN(xory, y0, uni, TF, main = descr)
lipoNN(xory, Ypred[[best]], uni, TF, col = 4, lwd = 4)
par(op)
}
##Â ===========================
## DETACH ZZ - END OF THE LOOP
##Â ===========================
detach(ZZ)
}




































Results
dfr0 <- getTimer(timer)
dfr <- data.frame(
ds_pkg.fun_algo = stringr::str_sub(dfr0[ ,1], 1, -4),
run = stringr::str_sub(dfr0[ ,1], -2, -1),
dataset = stringr::str_replace_all(stringr::str_extract(dfr0[, 1], pattern = "^\\w*_"), fixed("_"), ""),
method = stringr::str_replace_all(stringr::str_extract(dfr0[, 1], pattern = "_\\w*_"), fixed("_"), ""),
Elapsed = round(dfr0[ ,4], 5),
params = dfr0$params,
dfr0[, c("RMSE","MAE")]
)
dfr
Best Results
dfr %>%
group_by(dataset, method) %>%
summarise(minRMSE = min(RMSE), meanRMSE = mean(RMSE), meanTime = mean(Elapsed)) %>%
kable() %>%
kable_styling(bootstrap_options = c("striped", "hover", "condensed", "responsive")) %>%
collapse_rows(columns = 1:2, valign = "top")
| dataset |
method |
minRMSE |
meanRMSE |
meanTime |
| mDette |
adam |
3.0948 |
3.97626 |
0.008 |
| rmsprop |
2.8776 |
3.36848 |
0.010 |
| sgd |
4.9184 |
6.50956 |
0.028 |
| mFriedman |
adam |
0.0808 |
0.11418 |
0.012 |
| rmsprop |
0.0591 |
0.09248 |
0.010 |
| sgd |
0.1261 |
0.13998 |
0.010 |
| mIshigami |
adam |
2.7179 |
2.76180 |
0.014 |
| rmsprop |
2.3168 |
2.67236 |
0.020 |
| sgd |
2.9226 |
3.07040 |
0.008 |
| mRef153 |
adam |
6.0301 |
7.10828 |
0.010 |
| rmsprop |
4.6908 |
5.95992 |
0.006 |
| sgd |
7.5511 |
8.13686 |
0.006 |
| uDmod1 |
adam |
0.5276 |
0.57870 |
0.006 |
| rmsprop |
0.4809 |
0.51358 |
0.008 |
| sgd |
0.5673 |
0.69250 |
0.006 |
| uDmod2 |
adam |
0.3298 |
0.48506 |
0.004 |
| rmsprop |
0.3885 |
0.44276 |
0.006 |
| sgd |
0.4324 |
0.48176 |
0.010 |
| uDreyfus1 |
adam |
0.6140 |
0.68476 |
0.004 |
| rmsprop |
0.4876 |
0.59500 |
0.000 |
| sgd |
0.6968 |
0.80534 |
0.002 |
| uDreyfus2 |
adam |
0.6809 |
0.90494 |
0.004 |
| rmsprop |
0.4592 |
0.52160 |
0.000 |
| sgd |
0.6762 |
0.73370 |
0.004 |
| uGauss1 |
adam |
26.7491 |
27.76072 |
0.010 |
| rmsprop |
22.7846 |
25.61874 |
0.004 |
| sgd |
27.5268 |
28.25596 |
0.004 |
| uGauss2 |
adam |
19.2962 |
21.46538 |
0.008 |
| rmsprop |
13.2070 |
14.95164 |
0.016 |
| sgd |
17.3750 |
23.07154 |
0.002 |
| uGauss3 |
adam |
17.2794 |
20.72808 |
0.004 |
| rmsprop |
12.5749 |
14.10714 |
0.004 |
| sgd |
23.3915 |
27.76456 |
0.006 |
| uNeuroOne |
adam |
0.9157 |
1.02970 |
0.002 |
| rmsprop |
0.8519 |
0.95704 |
0.004 |
| sgd |
0.9877 |
1.09072 |
0.002 |
clearNN(donotremove)
LS0tCnRpdGxlOiAiTk5iZW5jaG1hcmsgfCBBTk4yIgphdXRob3I6IEFrc2hhaiBWZXJtYQpvdXRwdXQ6IGh0bWxfbm90ZWJvb2sKLS0tCgoKYGBge3IgbWVzc2FnZT1GQUxTRSwgd2FybmluZz1GQUxTRX0KbGlicmFyeShOTmJlbmNobWFyaykKbGlicmFyeShBTk4yKQoKbGlicmFyeShzdHJpbmdyKQpsaWJyYXJ5KGRwbHlyKQpsaWJyYXJ5KGthYmxlRXh0cmEpCmBgYAoKIyMgU2V0IEVudmlyb25tZW50CmBgYHtyfQpvcHRpb25zKHNjaXBlbiA9IDk5OTkpCm9wdGlvbnMoImRpZ2l0cy5zZWNzIiA9IDIpCnRpbWVyICA8LSBjcmVhdGVUaW1lcih2ZXJib3NlID0gRkFMU0UpCmBgYAoKIyMgRGF0YXNldHMgdG8gVGVzdAoKYGBge3J9Ck5OZGF0YVN1bW1hcnkoTk5kYXRhc2V0cykKYGBgCgojIyBOTiBUcmFpbiBGdW5jdGlvbgpgYGB7cn0KaHlwZXJQYXJhbXMgPC0gZnVuY3Rpb24ob3B0aW1fbWV0aG9kKSB7CiAgICAKICAgIGlmICghaXMuZWxlbWVudChvcHRpbV9tZXRob2QsIGMoInNnZCIsICJhZGFtIiwgInJtc3Byb3AiKSkpIHN0b3AoIkludmFsaWQgUGFyYW1ldGVycy4iKQogICAgaWYgKG9wdGltX21ldGhvZCA9PSAic2dkIikgeyBpdGVyIDwtIDEwOyBsciA8LSAwLjAxOyBwYXJhbXMgPC0gcGFzdGUwKCJtZXRob2Q9Iiwgb3B0aW1fbWV0aG9kLCAiXyIsICJscj0iLCBsciwgIl8iLCAiaXRlcj0iLCBpdGVyKX0gCiAgICBpZiAob3B0aW1fbWV0aG9kID09ICJhZGFtIikgeyBpdGVyIDwtIDE1OyBsciA8LSAwLjAyOyBwYXJhbXMgPC0gcGFzdGUwKCJtZXRob2Q9Iiwgb3B0aW1fbWV0aG9kLCAiXyIsICJscj0iLCBsciwgIl8iLCAiaXRlcj0iLCBpdGVyKX0gCiAgICBpZiAob3B0aW1fbWV0aG9kID09ICJybXNwcm9wIikgeyBpdGVyIDwtIDIwOyBsciA8LSAwLjAzOyBwYXJhbXMgPC0gcGFzdGUwKCJtZXRob2Q9Iiwgb3B0aW1fbWV0aG9kLCAiXyIsICJscj0iLCBsciwgIl8iLCAiaXRlcj0iLCBpdGVyKX0gCiAgICAKICAgIHBhcmFtcyA8LSBwYXN0ZTAoIm1ldGhvZD0iLCBvcHRpbV9tZXRob2QsICJfIiwgImxyPSIsIGxyLCAiXyIsICJpdGVyPSIsIGl0ZXIpCiAgICAKICAgIG91dCA8LSBsaXN0KGl0ZXIgPSBpdGVyLCBsciA9IGxyLCBwYXJhbXMgPSBwYXJhbXMpCiAgICAKICAgIHJldHVybiAob3V0KQp9CgoKCk5OdHJhaW4gPC0gZnVuY3Rpb24oeCwgeSwgaGlkZGVuX25ldXIsIG9wdGltX21ldGhvZCkgewogICAgCiAgICBoeXBlcl9wYXJhbXMgPC0gaHlwZXJQYXJhbXMob3B0aW1fbWV0aG9kKQogICAgCiAgICBpdGVyIDwtIGh5cGVyX3BhcmFtcyRpdGVyCiAgICBsciA8LSBoeXBlcl9wYXJhbXMkbHIKICAgIAogICAgTk5yZWcgPC0gbmV1cmFsbmV0d29yayhYID0geCwgeSA9IHksIAogICAgICAgICAgICAgICAgICAgICAgICAgICB2YWwucHJvcCA9IDAsIAogICAgICAgICAgICAgICAgICAgICAgICAgICBzdGFuZGFyZGl6ZSA9IEZBTFNFLCAKICAgICAgICAgICAgICAgICAgICAgICAgICAgaGlkZGVuLmxheWVycyA9IGMoaGlkZGVuX25ldXIpLCAKICAgICAgICAgICAgICAgICAgICAgICAgICAgcmVncmVzc2lvbiA9IFRSVUUsCiAgICAgICAgICAgICAgICAgICAgICAgICAgIGxvc3MudHlwZSA9ICJzcXVhcmVkIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgbi5lcG9jaHMgPSBpdGVyLAogICAgICAgICAgICAgICAgICAgICAgICAgICBvcHRpbS50eXBlID0gb3B0aW1fbWV0aG9kLAogICAgICAgICAgICAgICAgICAgICAgICAgICBsZWFybi5yYXRlcyA9IGxyLAogICAgICAgICAgICAgICAgICAgICAgICAgICB2ZXJib3NlID0gRkFMU0UsCiAgICAgICAgICAgICAgICAgICAgICAgICAgIHJhbmRvbS5zZWVkID0gYXMuaW50ZWdlcihydW5pZigxKSoxMDAwMDAwMCkpCiAgICAKICAgIHJldHVybiAoTk5yZWcpCn0KYGBgCgoKIyMgTWFpbiBMb29wCmBgYHtyIGZpZy5oZWlnaHQ9NSwgZmlnLndpZHRoPTE0fQpmb3IgKGRzZXQgaW4gbmFtZXMoTk5kYXRhc2V0cykpIHsKCiAgICAjI8KgPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09CiAgICAjI8KgRVhUUkFDVCBJTkZPUk1BVElPTiBGUk9NIFRIRSBTRUxFQ1RFRCBEQVRBU0VUCiAgICAjI8KgPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09CiAgICBkcyAgICAgPC0gTk5kYXRhc2V0c1tbZHNldF1dJGRzCiAgICBaICAgICAgPC0gTk5kYXRhc2V0c1tbZHNldF1dJFoKICAgIG5ldXIgICA8LSBOTmRhdGFzZXRzW1tkc2V0XV0kbmV1cgogICAgbnBhck5OIDwtIE5OZGF0YXNldHNbW2RzZXRdXSRucGFyTk4KICAgIGZtbGFOTiA8LSBOTmRhdGFzZXRzW1tkc2V0XV0kZm1sYU5OCiAgICBkb25vdHJlbW92ZSAgPC0gYygiZHNldCIsICJkc2V0cyIsICJkcyIsICJaIiwgIm5ldXIiLCAiVEYiLCAibnJlcCIsICJ0aW1lciIsCiAgICAgICAgICAgICAgICAgICAgICAiZG9ub3RyZW1vdmUiLCAiZG9ub3RyZW1vdmUyIikKICAgIGRvbm90cmVtb3ZlMiA8LSBjKCJkc2V0IiwgImRzZXRzIikgCgoKCiAgICAjI8KgPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09CiAgICAjIyBTRUxFQ1QgVEhFIEZPUk1BVCBSRVFVSVJFRCBCWSBUSEUgUEFDS0FHRS9BTEdPUklUSE0KICAgICMjIGQgPSBkYXRhLmZyYW1lLCBtID0gbWF0cml4LCB2ID0gdmVjdG9yL251bWVyaWMKICAgICMjwqBBVFRBQ0ggVEhFIE9CSkVDVFMgQ1JFQVRFRCAoeCwgeSwgWnh5LCAuLi4gKQogICAgIyMgPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09CiAgICBaWiAgICAgPC0gcHJlcGFyZVpaKFosIHhkbXYgPSAiZCIsIHlkbXYgPSAidiIsIHpkbSA9ICJkIiwgc2NhbGUgPSBUUlVFKQogICAgYXR0YWNoKFpaKQoKICAgICMjwqA9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0KICAgICMjwqBTRUxFQ1QgVEhFIFBBQ0tBR0UgVVNFRCBGT1IgVFJBSU5JTkcKICAgICMjIG5yZXAgPT4gU0VMRUNUIFRIRSBOVU1CRVIgT0YgSU5ERVBFTkRBTlQgUlVOUwogICAgIyPCoGl0ZXIgPT4gU0VMRUNUIFRIRSBNQVggTlVNQkVSIE9GIElURVJBVElPTlMKICAgICMjwqBURiAgID0+IFBMT1QgVEhFIFJFU1VMVFMKICAgICMjwqA9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0KCiAgICAKICAgIG5yZXAgICA8LSA1CiAgICBURiAgICAgPC0gVFJVRSAKCiAgICBtZXRob2QgPC0gYygic2dkIiwgImFkYW0iLCAicm1zcHJvcCIpCiAgICAgICAgCiAgICBmb3IgKG0gaW4gbWV0aG9kKSB7CiAgICAgICAgCiAgICAgICAgZGVzY3IgIDwtIHBhc3RlKGRzZXQsICJBTk4yOjpuZXVyYWxuZXR3b3JrIiwgbSwgc2VwID0gIl8iKQoKICAgICAgICAjI8KgQVVUTwogICAgICAgIFlwcmVkICA8LSBsaXN0KCkKICAgICAgICBSbXNlICAgPC0gbnVtZXJpYyhsZW5ndGggPSBucmVwKQogICAgICAgIE1hZSAgICA8LSBudW1lcmljKGxlbmd0aCA9IG5yZXApCiAgICAKICAgICAgICBmb3IoaSBpbiAxOm5yZXApewogICAgICAgICAgICBldmVudCAgICAgIDwtIHBhc3RlMChkZXNjciwgc3ByaW50ZigiXyUuMmQiLCBpKSkKICAgICAgICAgICAgdGltZXIkc3RhcnQoZXZlbnQpCiAgICAgICAgICAgICMjIyMgQURKVVNUIFRIRSBGT0xMT1dJTkcgTElORVMgVE8gVEhFIFBBQ0tBR0U6OkFMR09SSVRITQogICAgICAgICAgICAKICAgICAgICAgICAgaHlwZXJfcGFyYW1zIDwtIGh5cGVyUGFyYW1zKG9wdGltX21ldGhvZCA9IG0pCgogICAgICAgICAgICBOTnJlZyAgICAgIDwtIHRyeUNhdGNoKAogICAgICAgICAgICAgICAgICAgICAgICAgICAgTk50cmFpbih4ID0geCwgeSA9IHksIGhpZGRlbl9uZXVyID0gbmV1ciwgb3B0aW1fbWV0aG9kID0gbSksCiAgICAgICAgICAgICAgICAgICAgICAgICAgICBlcnJvciA9IGZ1bmN0aW9uKHkpIHtsbSh5IH4gMCwgZGF0YSA9IFp4eSl9CiAgICAgICAgICAgICAgICAgICAgICAgICAgKSAgICAgCiAgICAgICAgICAgIHlfcHJlZCAgICAgPC0gdHJ5Q2F0Y2goCiAgICAgICAgICAgICAgICAgICAgICAgICAgICB5bTAgKyB5c2QwKnByZWRpY3QoTk5yZWcsIHgpJHByZWRpY3Rpb25zLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgZXJyb3IgPSB5bTAKICAgICAgICAgICAgICAgICAgICAgICAgICApICAgICAKICAgICAgICAgICAgIyMjIwogICAgICAgICAgICBZcHJlZFtbaV1dIDwtIHlfcHJlZAogICAgICAgICAgICBSbXNlW2ldICAgIDwtIGZ1blJNU0UoeV9wcmVkLCB5MCkKICAgICAgICAgICAgTWFlW2ldICAgICA8LSBmdW5NQUUoeV9wcmVkLCB5MCkKICAgICAgICAgICAgdGltZXIkc3RvcChldmVudCwgUk1TRSA9IFJtc2VbaV0sIE1BRSA9IE1hZVtpXSwgcGFyYW1zID0gaHlwZXJfcGFyYW1zJHBhcmFtcywgcHJpbnRtc2cgPSBGQUxTRSkKICAgICAgICB9CiAgICAgICAgYmVzdCA8LSB3aGljaChSbXNlID09IG1pbihSbXNlLCBuYS5ybSA9IFRSVUUpKVsxXQogICAgICAgIGJlc3QgOyBSbXNlW1tiZXN0XV0KICAgICAgICAKICAgICAgICAjIyA9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0KICAgICAgICAjI8KgUExPVCBBTEwgTU9ERUxTIEFORCBUSEUgTU9ERUwgV0lUSCBUSEUgQkVTVCBSTVNFCiAgICAgICAgIyPCoHBhciBPUFRJT05TIENBTiBCRSBJTVBST1ZFRCBGT1IgQSBCRVRURVIgRElTUExBWQogICAgICAgICMjID09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PQogICAgICAgIG9wIDwtIHBhcihtZmNvbCA9IGMoMSwyKSkKICAgICAgICBwbG90Tk4oeG9yeSwgeTAsIHVuaSwgVEYsIG1haW4gPSBkZXNjcikKICAgICAgICBmb3IgKGkgaW4gMTpucmVwKSBsaXBvTk4oeG9yeSwgWXByZWRbW2ldXSwgdW5pLCBURiwgY29sID0gaSwgbHdkID0gMSkKICAgICAgICAKICAgICAgICBwbG90Tk4oeG9yeSwgeTAsIHVuaSwgVEYsIG1haW4gPSBkZXNjcikKICAgICAgICBsaXBvTk4oeG9yeSwgWXByZWRbW2Jlc3RdXSwgdW5pLCBURiwgY29sID0gNCwgbHdkID0gNCkKICAgICAgICBwYXIob3ApCiAgICB9CgoKIyPCoD09PT09PT09PT09PT09PT09PT09PT09PT09PQojIyBERVRBQ0ggWlogLSBFTkQgT0YgVEhFIExPT1AKIyPCoD09PT09PT09PT09PT09PT09PT09PT09PT09PQogICAgZGV0YWNoKFpaKQp9CmBgYAoKCiMjIFJlc3VsdHMKCmBgYHtyfQpkZnIwIDwtIGdldFRpbWVyKHRpbWVyKSAKCmRmciAgPC0gZGF0YS5mcmFtZSgKICAgIGRzX3BrZy5mdW5fYWxnbyA9IHN0cmluZ3I6OnN0cl9zdWIoZGZyMFsgLDFdLCAxLCAtNCksCiAgICBydW4gICAgID0gc3RyaW5ncjo6c3RyX3N1YihkZnIwWyAsMV0sIC0yLCAtMSksCiAgICBkYXRhc2V0ID0gc3RyaW5ncjo6c3RyX3JlcGxhY2VfYWxsKHN0cmluZ3I6OnN0cl9leHRyYWN0KGRmcjBbLCAxXSwgcGF0dGVybiA9ICJeXFx3Kl8iKSwgZml4ZWQoIl8iKSwgIiIpLAogICAgbWV0aG9kID0gc3RyaW5ncjo6c3RyX3JlcGxhY2VfYWxsKHN0cmluZ3I6OnN0cl9leHRyYWN0KGRmcjBbLCAxXSwgcGF0dGVybiA9ICJfXFx3Kl8iKSwgZml4ZWQoIl8iKSwgIiIpLAogICAgRWxhcHNlZCA9IHJvdW5kKGRmcjBbICw0XSwgNSksCiAgICBwYXJhbXMgPSBkZnIwJHBhcmFtcywKICAgIGRmcjBbLCBjKCJSTVNFIiwiTUFFIildCikKCgpkZnIKYGBgCgojIyBCZXN0IFJlc3VsdHMKCmBgYHtyfQpkZnIgJT4lCiAgICBncm91cF9ieShkYXRhc2V0LCBtZXRob2QpICU+JQogICAgc3VtbWFyaXNlKG1pblJNU0UgPSBtaW4oUk1TRSksIG1lYW5STVNFID0gbWVhbihSTVNFKSwgbWVhblRpbWUgPSBtZWFuKEVsYXBzZWQpKSAlPiUKICAgIGthYmxlKCkgJT4lCiAgICBrYWJsZV9zdHlsaW5nKGJvb3RzdHJhcF9vcHRpb25zID0gYygic3RyaXBlZCIsICJob3ZlciIsICJjb25kZW5zZWQiLCAicmVzcG9uc2l2ZSIpKSAlPiUKICAgIGNvbGxhcHNlX3Jvd3MoY29sdW1ucyA9IDE6MiwgdmFsaWduID0gInRvcCIpCmBgYAoKYGBge3J9CmNsZWFyTk4oZG9ub3RyZW1vdmUpCmBgYAoK